/*
 * Copyright 2011 Carnegie Mellon University
 * Licensed under the Apache License, Version 2.0 (the "License"); 
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *  
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, 
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied. See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */
package edu.cmu.lti.ritesdk;

import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * Evaluates the system output labels against the gold standard labels.
 * 
 * @author Hideki Shima
 *
 */
public class Evaluator {

  //For experimental purpose
  //When true, randomize system labels & conf
  private static boolean random = false; 
  
  public static String INVALID_LABEL = "N/A";
  
  public final static String ACCURACY = "ACCURACY";
  
  private static NumberFormat nf4 = NumberFormat.getInstance();
  private static NumberFormat nf6 = NumberFormat.getInstance();
  static {
    nf4.setMaximumFractionDigits(4);
    nf4.setMinimumFractionDigits(4);
    nf6.setMaximumFractionDigits(6);
    nf6.setMinimumFractionDigits(6);
  }
  
  public static EvaluationResult evaluate( AnalyzedDataSet d, Task task ) {
    String matrix = buildConfusionMatrix(d,task)+"\n";

    StringBuilder details = new StringBuilder("---- Evaluation Log ----\n\n");
    details.append("Legend: ID [SPACE] true_label [SPACE] system_label [SPACE] judgment\n\n");
    
    int numCorrect = 0;
    for ( String id : d.keySet() ) {
      AnalyzedTextPair t = d.get(id);
      String judgment;
      String gold;
      if (t.getTextPair()==null) {
        gold = "N/A";
        judgment = "SKIPPED EVALUATION";
      } else {
        gold = t.getTextPair().getGoldStandardLabel().toUpperCase();
        String system; 
        if (random) {
          system = Math.random()>0.5?"Y":"N";
        } else {
          system = t.getSystemLabel();
        }
        boolean isCorrect = system.toUpperCase().equals(gold);
        if (isCorrect) {
          numCorrect++;
          judgment = "CORRECT";
        } else {
          judgment = "WRONG";
        }        
      }
      details.append(id+" "+gold+" "+t.getSystemLabel()+" "+judgment+"\n");
    }
    double acc = (double)numCorrect/(double)d.size();
    StringBuilder sbScore = new StringBuilder();
    sbScore.append("---- Score ----\n\n");
    sbScore.append("Accuracy = "+nf4.format(acc)+"\n\n");
    Map<String,String> score = new LinkedHashMap<String,String>();
    score.put(ACCURACY, nf4.format(acc));

    return new EvaluationResult( score, matrix+sbScore, sbScore+matrix+details );
  }
  
  public static boolean haveLabels(AnalyzedDataSet d) {
    int numGoldLabel = 0;
    StringBuilder sb = new StringBuilder();
    for ( String id : d.keySet() ) {
      AnalyzedTextPair t = d.get(id);
      if (t.getTextPair()!=null 
              && t.getTextPair().getGoldStandardLabel()!=null 
              && t.getTextPair().getGoldStandardLabel().length()>0) {
        numGoldLabel++;
      } else {
        sb.append( "Gold standard label does not exist: "+id+"\n" );
      }
    }
    if (numGoldLabel>0 && sb.length()>0) {
      System.out.println(sb);
    }
    return numGoldLabel > 0;
  }
  
  private static String buildConfusionMatrix( AnalyzedDataSet d, Task task ) {
    List<String> labels = task.getDefinedLabels();
    List<String> validLabels = new ArrayList<String>(labels);
    validLabels.remove(INVALID_LABEL);
    boolean invalidLabelAppears = false;
    
    int N = labels.size();
    Map<String,Integer> index = new HashMap<String,Integer>();
    for ( int i=0; i<N; i++ ) {
      index.put(labels.get(i), i);
    }
    int numInvalidLabel = 0;
    int[][] matrix = new int[N][N];
    for ( String id : d.keySet() ) {
      AnalyzedTextPair t = d.get(id);
      Integer sIndex = index.get(t.getSystemLabel().toUpperCase());
      if ( t.getTextPair()==null ) {
        System.out.println("Skip evaluating "+id);
        continue;
      }
      Integer gIndex = index.get(t.getTextPair().getGoldStandardLabel().toUpperCase());
      if (gIndex==null) {
        System.out.println("ERROR: Invalid gold standard label \""+t.getTextPair().getGoldStandardLabel()+"\" has been identified at id="+t.getTextPair().getId()+". Allowed labels for "+task.toString()+": "+validLabels);
        System.out.println("Evaluaion mode is disabled.");
        System.exit(-1);
      }
      if (sIndex == null) {
        invalidLabelAppears = true;
        sIndex = index.get(INVALID_LABEL);
        if (!t.getSystemLabel().equals("") ) {
          System.out.println("ERROR: Invalid system label \""+t.getSystemLabel()+"\" has been returned at id="+t.getTextPair().getId()+". Allowed labels for "+task.toString()+": "+validLabels);
        } else {
          numInvalidLabel++;
        }
      }
      matrix[sIndex][gIndex]++;
    }
    if (numInvalidLabel>0) {
      System.out.println("ERROR: no labels returned for "+numInvalidLabel+" pairs.");
    }

    // Show N/A column/row only when invalid label appeared
    if (!invalidLabelAppears) {
      labels = validLabels;
      N = validLabels.size();
    }
    
    StringBuilder horizontal = new StringBuilder("----+");
    for ( int i=0; i<N; i++ ) {
      horizontal.append( "-----" );
    }
    horizontal.append("+-----\n");
    StringBuilder sb = new StringBuilder("---- Confusion Matrix ----\n\n");
    sb.append("Cols: true labels\n");
    sb.append("Rows: system labels\n\n");
    sb.append("    |");
    for ( String label : labels ) {
      sb.append(" "+String.format("%3s", label)+" ");
    }
    sb.append("|\n");
    sb.append(horizontal);
    int[] sums = new int[N];
    for ( int i=0; i<N; i++ ) {
      sb.append(" "+String.format("%-3s", labels.get(i))+"|");
      int sum = 0;
      for ( int j=0; j<N; j++ ) {
        sb.append(String.format("%4s", matrix[i][j])+" ");
        sum += matrix[i][j];
        sums[j] += matrix[i][j];
      }
      sb.append("| "+String.format("%3s", sum)+"\n");
    }
    sb.append(horizontal);
    sb.append("    |");
    for ( int i=0; i<N; i++ ) {
      sb.append(String.format("%4s", sums[i])+" ");
    }
    sb.append("|"+"\n");
    
    return sb.toString();
  }
  
}
